{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "dbb6ed90-cc1f-4966-a091-432cf5a12edf",
   "metadata": {},
   "outputs": [],
   "source": [
    "from tokenizers import Tokenizer\n",
    "#然后我们可以使用from_file() 方法从该文件里重新加载 Tokenizer 对象：\n",
    "new_tokenizer = Tokenizer.from_file(\"human2_formal.json\")\n",
    "#或者下面方法\n",
    "from transformers import GPT2TokenizerFast\n",
    "tokenizer = GPT2TokenizerFast(tokenizer_object=new_tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "af6f934c-fa35-4946-ae65-5e59d3fd0d8d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('human_token/tokenizer_config.json',\n",
       " 'human_token/special_tokens_map.json',\n",
       " 'human_token/vocab.json',\n",
       " 'human_token/merges.txt',\n",
       " 'human_token/added_tokens.json',\n",
       " 'human_token/tokenizer.json')"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.save_pretrained(\"human_token\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "039ed847-f5e7-487d-a1f4-f6528e228bea",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import AutoTokenizer, AutoModel\n",
    "model = AutoModel.from_pretrained('mygpt_formal')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "e18cd898-0c80-49eb-b990-1aa6c174aec2",
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save_pretrained(\"human_gpt_model\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "ffc50bb6-a491-461f-a65f-5b68121f92eb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('human_gpt2-v1/tokenizer_config.json',\n",
       " 'human_gpt2-v1/special_tokens_map.json',\n",
       " 'human_gpt2-v1/vocab.json',\n",
       " 'human_gpt2-v1/merges.txt',\n",
       " 'human_gpt2-v1/added_tokens.json',\n",
       " 'human_gpt2-v1/tokenizer.json')"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.save_pretrained(\"human_gpt2-v1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "988bc92e-1944-4800-b2f5-b4f43356bfe4",
   "metadata": {},
   "outputs": [],
   "source": [
    "model.save_pretrained(\"human_gpt2-v1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "580fb0a6-4cb6-4beb-88b9-cd121dea74c6",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
